package com._21cn.fetch.utils;

import com._21cn.cms.article.entity.Article;
import com._21cn.cms.article.utils.FormatContentUtil;
import com._21cn.framework.http.HttpCaller;
import com._21cn.framework.utils.StringUtil;
import com.alibaba.fastjson.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.*;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.regex.Pattern;

public class FetchQiaoli {
    private static String postUrl = "http://admin.cms.qiaoliba.com/admin/article/article/save.do";

    private String linkPattern = "(.*)\\d+_\\d+.html";
    private Set<String> linkSet = new TreeSet();

    public Document getDocument(String url) {
        HttpCaller httpCaller = HttpCaller.get(url);
        httpCaller.addHeader("Referer","www.qiaoli.org");
        httpCaller.addHeader("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3514.0 Safari/537.36");
        String html = httpCaller.call().getString();
        Document doc = Jsoup.parse(html);
        return doc;
    }

    public Set<String> getPager(Document document) {
        Elements itemElements = document.select(".artsli-wrap").select("ul").select("a");
        for (Element element : itemElements) {
            String link = element.attr("href");
            if (Pattern.matches(this.linkPattern, link)) {
                this.linkSet.add(link);
            }
        }
        return this.linkSet;
    }

    public Article getArticle(String contentUrl) {
        Document doc = getDocument(contentUrl);
        Article article = formatArticle(doc);
        article.setContent(""); //pager第一页有冲突
        Set<String> pager = getPager(doc);
        for (String url : pager) {
//            int index = contentUrl.lastIndexOf("/");
//            String before = contentUrl.substring(0, index + 1);
            url = "https://www.qiaoli.org" + "/" + url;
            Document document = getDocument(url);
            String content = getContent(document);
            if (!StringUtil.isEmpty(content)) {
                content = article.getContent() + "<p class=\"pager\">&nbsp;</p>" + content;
                article.setContent(content);
            }
        }
        return article;
    }

    public Article formatArticle(Document doc) {
        Article article = new Article();
        String title = doc.select(".arti-head").select("h2").text();
        String content = getContent(doc);
        article.setTitle(title);
        article.setContent(content);
        return article;
    }

    public String getContent(Document doc){
        String content = FormatContentUtil.formatContent(doc.select(".arti-content").html()).replaceAll("<div>", "").replaceAll("</div>", "");
        return content;
    }

    public Map<String, String> extractPictureListFromContent(String content) {
        String fetUrl = "http://admin.cms.qiaoliba.com/pictureFetch.do?status=1";
        Map resultMap = new HashMap();
        String sourceUrl;
        JSONObject array;
        if (!StringUtil.isEmpty(content)) {
            Document doc = Jsoup.parse(content);
            Elements imgs = doc.getElementsByTag("img");
            int i = 0;
            String urls = "";
            for (Element img : imgs) {
                sourceUrl = img.attr("src").trim();
                urls = urls + ",21cn.com," + sourceUrl;
            }
            if (!StringUtil.isEmpty(urls)) {
                urls = urls.substring(",21cn.com,".length(), urls.length());
            }
            String json = HttpCaller.post(fetUrl).addParam("url", urls).call().getString();
            array = JSONObject.parseObject(json);
            for (Element img : imgs) {
                sourceUrl = img.attr("src").trim();
                JSONObject obj = array.getJSONObject(sourceUrl);
                String url = obj.getString("url");

                resultMap.put(sourceUrl, url);
            }
        }
        return resultMap;
    }

    public static void saveArticle(String line) {
        Article article = new FetchQiaoli().getArticle(line);
        String content = article.getContent();
        Map<String, String> urlmap = new FetchQiaoli().extractPictureListFromContent(content);
        for (String url : urlmap.keySet()) {
            content = content.replace(url, (CharSequence) urlmap.get(url));
        }

        System.out.println(content);
        HttpCaller.post(postUrl).addParam("title", StringUtil.utf82iso(article.getTitle()))
                .addParam("content", StringUtil.utf82iso(content)).addParam("sourceName", StringUtil.utf82iso("俏丽女人"))
                .addParam("mainColumnId", "24")
                .addParam("publishStatus", "2")
                .addHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
                .addHeader("Accept-Encoding", "gzip, deflate")
                .addHeader("Content-Type", "application/x-www-form-urlencoded")
                .addHeader("Cookie", "admin_auth_key=xxt:value_admin")
                .call();
        System.out.println("finish url:" + line);
    }

    public static void main(String[] args) {
        BufferedReader br = null;
        try {
            br = new BufferedReader(new InputStreamReader(new FileInputStream("D:/article.txt"), "UTF-8"));
        } catch (UnsupportedEncodingException e) {
            e.printStackTrace();
        } catch (FileNotFoundException e) {
            e.printStackTrace();
        }
        String line = "";
        try {
            while ((line = br.readLine()) != null)
                saveArticle(line);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}